import urllib.request
from bs4 import BeautifulSoup
from selenium import webdriver
from selenium.webdriver.common.action_chains import ActionChains
from time import sleep
from selenium.webdriver.common.by import By
import os
from dbhelper import DBHelper


def get_html(url):
	# response = urllib.request.urlopen(url)
	# if response.getcode() != 200:
	# 	return None
	# return response.read()
	response_text = ""
	html = open('./test_index.html','r',encoding="utf-8")
	response_text = html.read()
	# print(response_text)
	return response_text


def html_parse(response_text):
	if response_text is None:
		return
	soup = BeautifulSoup(response_text, 'html.parser')
	category_info = category_content_parse(soup)
	# return
	category_tab_node = soup.find('div', class_='category-tab-content')
	category_arr = []
	category_tab_list = category_tab_node.find_all('li')
	index = 0
	for tab in category_tab_list:
		category_object = {}
		tab_items = []
		tab_items_node = tab.find_all('a')
		for item in tab_items_node:
			tab_items.append(item.get_text())
		category_object['tab'] = tab_items
		category_object['tab_info'] = category_info[index]
		category_arr.append(category_object)
		index = index + 1
	print(category_arr)
	return category_arr

def category_content_parse(soup):
	category_contents = soup.find_all('div', class_='pannel-con j_CategoryMenuPannel')
	category_info = []
	for category in category_contents:
		hot_words = category.find_all('div', class_='hot-word-line')
		hot_word_obj = {}
		for hot_word in hot_words:
			hot_word_text = hot_word.find('div', class_='line-title').find('div', class_='title-text').get_text()
			# print(hot_word_text)
			hot_word_arr = []
			hot_word_items = hot_word.find('div', class_='line-con').find_all('a')
			for item in hot_word_items:
				hot_word_arr.append(item.get_text())
			hot_word_obj[hot_word_text] = hot_word_arr
		category_info.append(hot_word_obj)

	# print(category_info)
	return category_info

def save_category_in_db(category_arr, dbHelper):
	index = 0
	for category_item in category_arr:
		sql = "insert into nav_index(nav_index) values (%s)"
		params = ("item-%d"%(index))
		# print("item-%d"%(index))
		# return
		nav_id = dbHelper.insert(sql, params)

		nav_normal_arr = category_item['tab']
		for normal_item in nav_normal_arr:
			sql = "insert into nav_normal(nav_index_id,nav_normal_item) values (%s,%s)"
			params = (nav_id, normal_item)
			dbHelper.insert(sql, *params)

		nav_normal_info_obj = category_item['tab_info']
		for key in nav_normal_info_obj:

			sql = "insert into nav_normal_title(nav_index_id,nav_normal_title_text) values (%s,%s)"
			params = (nav_id, key)
			nav_normal_title_id = dbHelper.insert(sql, *params)

			val = nav_normal_info_obj[key]
			for hot_word in val:
				sql = "insert into nav_hot_word(nav_normal_title_id,nav_hot_word) values (%s,%s)"
				params = (nav_normal_title_id, hot_word)
				dbHelper.insert(sql, *params)

		index = index + 1


def main():
	tamll_url = r"https://www.tmall.com/"
	response_text = get_html(tamll_url)
	# print(response_text)
	category_arr = html_parse(response_text)

	mysql_connect_info = {
        'host' : '127.0.0.1',
        'port' : 3306,
        'user' : 'root',
        'password' : 'root',
        'db' : 'tbk_shop_mall'
	}

	dbHelper = DBHelper(mysql_connect_info)
	save_category_in_db(category_arr, dbHelper)

if __name__ == '__main__':
	main()